Sara Shirinkam (yzc280)¶

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import re
import string
import nltk
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('darkgrid')
import plotly.express as ex
import plotly.graph_objs as go
import plotly.offline as pyo
from plotly.subplots import make_subplots
pyo.init_notebook_mode()
from sklearn.decomposition import TruncatedSVD,PCA
from sklearn.feature_extraction.text import CountVectorizer
nltk.download('vader_lexicon')
from nltk.sentiment.vader import SentimentIntensityAnalyzer as SIA
from wordcloud import WordCloud,STOPWORDS
from pandas.plotting import autocorrelation_plot
from statsmodels.graphics.tsaplots import plot_acf
from statsmodels.graphics.tsaplots import plot_pacf
from statsmodels.tsa.seasonal import seasonal_decompose
from nltk.util import ngrams
from nltk import word_tokenize
from nltk.stem import PorterStemmer
from nltk.stem import WordNetLemmatizer
import random
plt.rc('figure',figsize=(17,13))

data = pd.read_csv(r'C:\vaccination_tweets.csv')
data.head(3)

data.info()

#Check the missing values for each column
data.isnull().sum()

#Add a new column to count the number of text keywords used
data['Num_Keywords'] = data['text'].str.len()

#Top 20 categories with most keyword
data[['text','Num_Keywords']].sort_values('Num_Keywords',ascending = False).head(10)

fig = plt.figure()
ax = fig.add_subplot(111)

x = data['Num_Keywords']
numBins = 100
ax.hist(x,numBins,color='green',alpha=0.7)
plt.show()

def clean(text):
    text = re.sub('https?://\S+|www\.\S+', '', text)
    text = re.sub(r'\s+', ' ', text, flags=re.I)
    text = re.sub('\[.*?\]', '', text)
    text = re.sub('\n', '', text)
    text = re.sub('\w*\d\w*', '', text)
    text = re.sub('<.*?>+', '', text)
    return text
    


data['text'] = data['text'].apply(lambda x:clean(x))
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
analyser = SentimentIntensityAnalyzer()
scores=[]
for i in range(len(data['text'])):
    
    score = analyser.polarity_scores(data['text'][i])
    score=score['compound']
    scores.append(score)
sentiment=[]
for i in scores:
    if i>=0.05:
        sentiment.append('Positive')
    elif i<=(-0.05):
        sentiment.append('Negative')
    else:
        sentiment.append('Neutral')
data['sentiment']=pd.Series(np.array(sentiment))
data.head()

sid = SIA()
data['sentiments']           = data['text'].apply(lambda x: sid.polarity_scores(' '.join(re.findall(r'\w+',x.lower()))))
data['Positive Sentiment']   = data['sentiments'].apply(lambda x: x['pos']+1*(10**-6)) 
data['Neutral Sentiment']    = data['sentiments'].apply(lambda x: x['neu']+1*(10**-6))
data['Negative Sentiment']   = data['sentiments'].apply(lambda x: x['neg']+1*(10**-6))

data.drop(columns=['sentiments'],inplace=True)
data.head()

import matplotlib.pyplot as plt 

print("Number of rows per star rating:")
print(data['sentiment'].value_counts())

# Function to map stars to sentiment
def map_sentiment(label):
    if label == 'Positive':
        return 1
    elif label == 'Negative':
        return -1
    else:
        return 0
# Mapping sentiment into numeric
data['sentiment1'] = [ map_sentiment(x) for x in data['sentiment']]
# Plotting the sentiment distribution
plt.figure()
pd.value_counts(data['sentiment1']).plot.bar(title="Sentiment distribution in df")
plt.xlabel("Sentiment")
plt.ylabel("No. of rows in data")
plt.show()

data.head()

Most_Positive = data[data['Positive Sentiment'].between(0.4,1)]
Most_Negative = data[data['Negative Sentiment'].between(0.25,1)]

Most_Positive_text = ' '.join(Most_Positive.text)
Most_Negative_text = ' '.join(Most_Negative.text)


pwc = WordCloud(width=600,height=400,collocations = False,background_color='white').generate(Most_Positive_text)
nwc = WordCloud(width=600,height=400,collocations = False,background_color='white').generate(Most_Negative_text)

plt.subplot(1,2,1)
plt.title('Common Words Among Most Positive Tweets',fontsize=16,fontweight='bold')
plt.imshow(pwc)
plt.axis('off')
plt.subplot(1,2,2)
plt.title('Common Words Among Most Negative Tweets',fontsize=16,fontweight='bold')
plt.imshow(nwc)
plt.axis('off')

plt.show()

b_date_count = data.groupby(by='date').count().reset_index()
b_date_count = b_date_count.rename(columns={'id':'Tweets per day'})
fig = ex.line(b_date_count,x='date',y='Tweets per day')

# fig.add_annotation(x=b_date_mean['date'].values[15], y=.024,
#             text=r"Start Of Incline",
#             showarrow=True,
#             arrowhead=6,
#             yshift=10)


# fig.add_shape(type="line",
#     x0=b_date_count['day'].values[0], y0=b_date_count['Negative Sentiment'].mean(), x1=b_date_count['day'].values[-1], y1=b_date_count['Negative Sentiment'].mean(),
#     line=dict(
#         color="Red",
#         width=2,
#         dash="dashdot",
#     ),
#         name='Mean',
# )

fig.add_shape(type="line",
    x0=b_date_count['date'].values[0], y0=b_date_count['sentiment'].mean(), x1=b_date_count['date'].values[-1], y1=b_date_count['sentiment'].mean(),
    line=dict(
        color="Red",
        width=2,
        dash="dashdot",
    ),
        name='Mean',
)

fig.update_traces(mode="markers+lines")
fig.update_layout(hovermode="x unified")

data_p=data[data['sentiment']=='Positive']
import datetime
b_date_count_p = data_p.groupby(by='date').count().reset_index()
b_date_count_p = b_date_count_p.rename(columns={'id':'Positive tweets per day'})
fig = ex.line(b_date_count_p,x='date',y='Positive tweets per day')
fig.update_traces(mode="markers+lines")
fig.update_layout(hovermode="x unified")


fig.add_shape(type="line",
    x0=b_date_count_p['date'].values[0], y0=b_date_count_p['Positive Sentiment'].mean(), x1=b_date_count_p['date'].values[-1], y1=b_date_count_p['Positive Sentiment'].mean(),
    line=dict(
        color="Red",
        width=2,
        dash="dashdot",
    ),
        name='Mean',
)

# importing webbrowser python module
import webbrowser
#Assigning URL to be opened
strURL = "https://www.reuters.com/business/healthcare-pharmaceuticals/pfizerbiontech-vaccine-appears-effective-against-mutation-new-coronavirus-2021-01-08/"
#Open url in default browser
webbrowser.open(strURL, new=2)

data_n=data[data['sentiment']=='Negative']
import datetime
b_date_count_n = data_n.groupby(by='date').count().reset_index()
b_date_count_n = b_date_count_n.rename(columns={'id':'Negative tweets per day'})
fig = ex.line(b_date_count_n,x='date',y='Negative tweets per day')
fig.update_traces(mode="markers+lines")
fig.update_layout(hovermode="x unified")


fig.add_shape(type="line",
    x0=b_date_count_n['date'].values[0], y0=b_date_count_n['Negative Sentiment'].mean(), x1=b_date_count_n['date'].values[-1], y1=b_date_count_n['Negative Sentiment'].mean(),
    line=dict(
        color="Red",
        width=2,
        dash="dashdot",
    ),
        name='Mean',
)

data_t=data[data['sentiment']=='Neutral']
import datetime
b_date_count_t = data_p.groupby(by='date').count().reset_index()
b_date_count_t = b_date_count_t.rename(columns={'id':'Neutral tweets per day'})
fig = ex.line(b_date_count_t,x='date',y='Neutral tweets per day')


fig.update_traces(mode="markers+lines")
fig.update_layout(hovermode="x unified")


fig.add_shape(type="line",
    x0=b_date_count_t['date'].values[0], y0=b_date_count_t['Neutral Sentiment'].mean(), x1=b_date_count_t['date'].values[-1], y1=b_date_count_t['Neutral Sentiment'].mean(),
    line=dict(
        color="Red",
        width=2,
        dash="dashdot",
    ),
        name='Mean',
)